package com.qianxun.spider;

import com.alibaba.fastjson.JSON;
import com.qianxun.spider.bean.SpiderBean;
import com.qianxun.spider.bean.SpiderBeanField;
import com.qianxun.spider.config.SpiderBeanConfig;
import com.qianxun.spider.config.SpiderConfig;
import com.qianxun.spider.enums.HttpMethod;
import com.qianxun.spider.http.*;
import com.qianxun.spider.monitor.SpiderMonitor;
import com.qianxun.spider.pipeline.SpiderPipeline;
import com.qianxun.spider.util.ObjectUtil;
import com.qianxun.spider.util.PatternUtil;
import com.qianxun.spider.util.scheduler.SchedulerUtil;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.logging.LogFactory;
import org.jsoup.Connection;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.select.Elements;
import org.quartz.SchedulerException;

import java.util.*;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

/**
 * @author: huwei
 * @date: 2020/6/28 15:05
 * @version: 1.0.0
 */
@Slf4j
public class Spider {
    private List<SpiderBeanConfig> configList;
    private TaskManager taskManager;
    private SpiderConfig spiderConfig;
    private Inner inner;
    private ThreadLocalData threadLocalData;
    private AtomicBoolean startFlag = new AtomicBoolean(false);

    private Spider() {
        this.inner = new Inner(this);
        this.configList = new ArrayList<>();
        this.taskManager = new TaskManager();
        this.spiderConfig = new SpiderConfig();
    }

    public static Spider newly() {
        return new Spider();
    }

    public static Object get(SpiderBean spiderBean, String url, Boolean... post) {
        Object[] res = new Object[]{null};
        SpiderBeanConfig spiderBeanConfig = SpiderBeanConfig.newly().spiderBean(spiderBean).urlTemplate(url)
                .pipeline((parseMap, parseObject, spiderBeanConfig1, spider, threadLocalData) -> res[0] = parseObject);
        HttpRequest httpRequest = post != null && post.length > 0 && post[0] ? new HttpPost(url) : new HttpGet(url);
        Spider spider = newly().setUrl(httpRequest).config(spiderBeanConfig);
        spider.start();
        spider.shutdown();
        return res[0];
    }

    public Spider debug(boolean debug) {
        this.spiderConfig.setDebug(debug);
        return this;
    }

    public Spider threadCount(int count) {
        this.spiderConfig.setThreadCount(count <= 0 ? 1 : count);
        return this;
    }

    public Spider intervalMillis(int intervalMillis) {
        this.spiderConfig.setIntervalMillis(intervalMillis);
        return this;
    }

    public Spider maxTaskCount(int count) {
        this.spiderConfig.setMaxTaskCount(count <= 0 ? this.spiderConfig.getMaxTaskCount() : count);
        return this;
    }

    public Spider config(SpiderConfig spiderConfig) {
        if (spiderConfig == null || spiderConfig.getThreadCount() <= 0) {
            throw new RuntimeException("配置错误");
        }
        this.spiderConfig = spiderConfig;
        return this;
    }

    public Spider config(SpiderBeanConfig spiderBeanConfig) {
        this.configList.add(spiderBeanConfig);
        return this;
    }

    public Spider config(List<SpiderBeanConfig> configList) {
        this.configList.addAll(configList);
        return this;
    }

    public Spider setUrl(String url) {
        return setUrl(url, false);
    }

    public Spider setUrl(String url, boolean postRequest) {
        return setUrl(postRequest ? new HttpPost(url) : new HttpGet(url));
    }

    public Spider setUrl(HttpRequest httpRequest) {
        return setUrl(Arrays.asList(httpRequest));
    }

    public Spider setUrl(List<HttpRequest> httpRequestList) {
        this.taskManager.addAll(httpRequestList);
        return this;
    }

    public List<HttpRequest> getUrlList() {
        return this.taskManager.requestList();
    }

    public int getUrlTotal() {
        return this.taskManager.requestCount();
    }

    public boolean urlContains(String url) {
        return this.taskManager.requestContains(url);
    }

    public boolean isStarted() {
        return this.startFlag.get();
    }

    public Spider shutdown() {
        //通知执行管道
        this.configList.forEach(spiderBeanConfig -> {
            spiderBeanConfig.getSpiderMonitorList().forEach(spiderMonitor -> spiderMonitor.close(spiderBeanConfig));
            spiderBeanConfig.getSpiderPipelineList().forEach(spiderPipeline -> spiderPipeline.close(spiderBeanConfig));
        });
        this.startFlag.set(false);
        this.taskManager.clear();
        this.threadLocalData.clear();
        return this;
    }

    public void start() {
        startBefore();

        HttpRequest request = null;
        while (this.startFlag.get() && (request = this.taskManager.pop()) != null) {
            //设置专属数据
            this.threadLocalData.setCurrentRequestUrl(request);
            this.threadLocalData.setCurrentRequestUrlDomain(PatternUtil.getUrlDomain(request.url()));

            //匹配当前请求的bean信息
            List<SpiderBeanConfig> list = this.inner.matchSpiderBeanConfig(request.url());

            Document document = this.inner.request(request ,list);
            if(document != null){
                //前置监听
                this.inner.parseBefore(list, document);
                try {
                    Elements elements = document.getAllElements();
                    Document finalDocument = document;
                    list.forEach(spiderBeanConfig -> {
                        Map<String, Object> parseData = this.inner.parseHtml(spiderBeanConfig, elements);
                        //后置监听
                        this.inner.parseAfter(list, finalDocument, parseData);

                        //管道处理
                        List<SpiderPipeline> pipelineList = spiderBeanConfig.getSpiderPipelineList();
                        if (pipelineList != null) {
                            Object obj = this.inner.switchParseObjectToBean(parseData, spiderBeanConfig.getSpiderBean().getBeanClass(), spiderBeanConfig.getSpiderBean().isCollector());
                            pipelineList.forEach(spiderPipeline -> spiderPipeline.process(parseData, obj, spiderBeanConfig, this, threadLocalData));
                        }
                    });
                } catch (Throwable throwable) {
                    System.err.println("爬虫任务解析数据异常,请求地址：" + request.url());
                    this.inner.parseError(list, document, throwable);
                }
            }
            
            //下班时间
            if (getUrlTotal() > 0 && this.spiderConfig.getIntervalMillis() > 0) {
                try {
                    Thread.sleep(this.spiderConfig.getIntervalMillis());
                } catch (InterruptedException e) {
                }
            }
        }
        shutdown();
        System.err.println("spider没有任务，可以休息了..");
    }

    public void syncStart() {
        for (int i = 0; i < this.spiderConfig.getThreadCount(); ++i) {
            try {
                SchedulerUtil.schedule(() -> {
                    try {
                        start();
                    } catch (Throwable throwable) {
                        log.error("爬虫任务执行异常", throwable);
                    }
                });
            } catch (SchedulerException e) {
                new Thread(() -> {
                    try {
                        start();
                    } catch (Throwable throwable) {
                        log.error("爬虫任务执行异常", throwable);
                    }
                }, "spider task" + i).start();
            }
        }
    }

    private void startBefore() {

        //日志
        if (this.spiderConfig.isDebug()) {
            // 设置默认工厂类
            System.setProperty("org.apache.commons.logging.LogFactory", "org.apache.commons.logging.impl.LogFactoryImpl");
            // 设置日志打印类
            LogFactory.getFactory().setAttribute("org.apache.commons.logging.Log", "org.apache.commons.logging.impl.SimpleLog");
            //设置默认日志级别
            LogFactory.getFactory().setAttribute("org.apache.commons.logging.simplelog.defaultlog", "debug");
        } else {
            // 设置默认工厂类
            System.setProperty("org.apache.commons.logging.LogFactory", "org.apache.commons.logging.impl.LogFactoryImpl");
            // 设置日志打印类
            LogFactory.getFactory().setAttribute("org.apache.commons.logging.Log", "org.apache.commons.logging.impl.SimpleLog");
            //设置默认日志级别
            LogFactory.getFactory().setAttribute("org.apache.commons.logging.simplelog.defaultlog", "error");
        }
        //线程数据初始
        this.threadLocalData = new ThreadLocalData();
        //开始
        this.startFlag.set(true);
    }

    private class TaskManager {
        private LinkedList<HttpRequest> requestList;
        private LinkedHashSet<String> urlDeWightSet;

        private TaskManager() {
            this.requestList = new LinkedList<>();
            this.urlDeWightSet = new LinkedHashSet<>();
        }

        private HttpRequest pop() {
            HttpRequest httpRequest = this.requestList.poll();
            if (httpRequest != null) {
                this.urlDeWightSet.remove(httpRequest.url());
            }
            return httpRequest;
        }

        public void addAll(List<HttpRequest> httpRequestList) {
            List<HttpRequest> temp = new ArrayList<>(httpRequestList.size());
            //去重 ,加锁
            synchronized (TaskManager.class) {
                httpRequestList.forEach(httpRequest -> {
                    if (!this.urlDeWightSet.contains(httpRequest.url())) {
                        temp.add(httpRequest);
                        this.urlDeWightSet.add(httpRequest.url());
                    }
                });
            }
            this.requestList.addAll(temp);
        }

        public List<HttpRequest> requestList() {
            return Collections.unmodifiableList(requestList);
        }

        public void clear() {
            this.requestList.clear();
        }

        public int requestCount() {
            return this.requestList.size();
        }

        public boolean requestContains(String url) {
            return this.urlDeWightSet.contains(url);
        }
    }

    /**
     * 线程专属数据
     */
    public class ThreadLocalData {
        private ThreadLocal<Map<String, Object>> localData = new InheritableThreadLocal<>();

        {
            this.localData.set(new HashMap<>(3));
        }

        public HttpRequest getCurrentRequestUrl() {
            return (HttpRequest) this.localData.get().get("currentRequestUrl");
        }

        public String getCurrentRequestUrlDomain() {
            return (String) this.localData.get().get("currentRequestUrlDomain");
        }

        public List<HttpProxy> getCurrentRequestHttpProxyList(){
            return (List<HttpProxy>) this.localData.get().get("currentUseHttpProxyList");
        }

        private ThreadLocalData setHttpProxy(HttpProxy httpProxy){
            List<HttpProxy> httpProxies = (List<HttpProxy>) this.localData.get().get("currentUseHttpProxyList");
            if(httpProxies == null){
                httpProxies = new ArrayList<>(3);
                this.localData.get().put("currentUseHttpProxyList" ,httpProxies);
            }
            httpProxies.add(httpProxy);
            return this;
        }

        private ThreadLocalData setCurrentRequestUrl(HttpRequest httpRequest) {
            this.localData.get().put("currentRequestUrl", httpRequest);
            return this;
        }

        private ThreadLocalData setCurrentRequestUrlDomain(String domain) {
            this.localData.get().put("currentRequestUrlDomain", domain);
            return this;
        }

        private void clear() {
            this.localData.remove();
        }
    }

    private class Inner {
        private Spider spider;

        public Inner(Spider spider) {
            this.spider = spider;
        }

        Document request(HttpRequest request ,List<SpiderBeanConfig> list){
            //请求结果
            Document document = null;
            Throwable requestError = null;

            //创建url请求连接对象并设置参数
            Connection connection = Jsoup.connect(request.url());
            connection.method(request.method() == HttpMethod.GET ? Connection.Method.GET : Connection.Method.POST);
            if(spiderConfig.getConnTimeoutMillis() != null && spiderConfig.getConnTimeoutMillis() > 0){
                connection.timeout(spiderConfig.getConnTimeoutMillis());
            }
            //前置监听
            this.requestBefore(list, connection);

            HttpProxyPool httpProxyPool = spiderConfig.getHttpProxyPool();
            if(httpProxyPool != null){
                if(httpProxyPool.localRetry() && httpProxyPool.localPriority()){
                    //优先使用本地服务
                    try {
                        document = connection.execute().parse();
                    }catch (Throwable throwable){
                        requestError = throwable;
                    }
                }
                if(document == null) {
                    int retry = httpProxyPool.retry() <= 0 ? 1 : httpProxyPool.retry();
                    HttpProxy httpProxy = null;
                    while (document == null && retry-- > 0 && (httpProxy = httpProxyPool.next()) != null) {
                        System.err.println("------------- >> 使用代理爬取(" + threadLocalData.getCurrentRequestUrl().url() + ")数据：： 代理信息：：" + httpProxy);
                        //设置代理
                        connection.proxy(httpProxy.getHost(), httpProxy.getPort());
                        try {
                            document = connection.execute().parse();
                        } catch (Throwable throwable) {
                            //计入代理中
                            threadLocalData.setHttpProxy(httpProxy);
                            requestError = throwable;
                            //标记该代理失败
                            httpProxyPool.makeFailure(httpProxy);
                        }
                    }
                }
            }
            //请求失败，再次使用本地尝试
            if(document == null && (httpProxyPool == null || (httpProxyPool.localRetry() && !httpProxyPool.localPriority()))){
                //重新配置数据
                connection = Jsoup.connect(request.url());
                connection.method(request.method() == HttpMethod.GET ? Connection.Method.GET : Connection.Method.POST);
                if(spiderConfig.getConnTimeoutMillis() != null && spiderConfig.getConnTimeoutMillis() > 0){
                    connection.timeout(spiderConfig.getConnTimeoutMillis());
                }
                try {
                    document = connection.execute().parse();
                }catch (Throwable throwable){
                    requestError = throwable;
                }
            }

            if(document == null){
                System.err.println("爬虫任务请求数据异常,请求地址：" + request.url());
                this.requestError(list, connection, requestError);
            }else{
                //后置监听
                this.requestAfter(list, connection);
            }
            return document;
        }

        List<SpiderBeanConfig> matchSpiderBeanConfig(String url) {
            List<SpiderBeanConfig> res = new ArrayList<>();

            List<SpiderBeanConfig> temp = Collections.unmodifiableList(configList);
            for (int i = 0; i < temp.size(); ++i) {
                SpiderBeanConfig spiderBeanConfig = temp.get(i);
                if (spiderBeanConfig.getUrlTemplate().equals("*") || spiderBeanConfig.getUrlTemplate().equals(url)) {
                    res.add(spiderBeanConfig);
                } else {
                    List<String> curUriList = spiltUri(url);
                    List<String> configUriList = spiltUri(spiderBeanConfig.getUrlTemplate());
                    if (curUriList.isEmpty() || configUriList.isEmpty() || configUriList.size() != curUriList.size()) {
                        continue;
                    }
                    boolean math = true;
                    for (int j = 0; j < configUriList.size(); ++j) {
                        String configUri = configUriList.get(j);
                        String curUri = curUriList.get(j);
                        if (!configUri.equals(curUri)) {
                            if (configUri.startsWith("{") && configUri.endsWith("}")) {
                                String patternUrl = configUri.substring(1, configUri.length() - 1);
                                if (!"*".equals(patternUrl)) {
                                    Pattern pattern = Pattern.compile("^" + configUri + "$");
                                    Matcher matcher = pattern.matcher(curUri);
                                    if (!matcher.find()) {
                                        math = false;
                                        break;
                                    }
                                }
                            } else {
                                math = false;
                                break;
                            }
                        }
                    }
                    if (math) {
                        res.add(spiderBeanConfig);
                    }
                }
            }
            return res;
        }

        List<String> spiltUri(String url) {
            String[] uris = url.substring(url.indexOf("://") + 3).split("/");
            return Arrays.asList(uris);
        }

        void requestBefore(List<SpiderBeanConfig> list, Connection connection) {
            list.forEach(spiderBeanConfig -> {
                List<SpiderMonitor> monitorList = spiderBeanConfig.getSpiderMonitorList();
                if (monitorList != null) {
                    monitorList.forEach(spiderMonitor -> spiderMonitor.requestBefore(connection, spiderBeanConfig, spider, threadLocalData));
                }
            });
        }

        void requestError(List<SpiderBeanConfig> list, Connection connection, Throwable throwable) {
            list.forEach(spiderBeanConfig -> {
                List<SpiderMonitor> monitorList = spiderBeanConfig.getSpiderMonitorList();
                if (monitorList != null) {
                    monitorList.forEach(spiderMonitor -> spiderMonitor.requestError(connection, spiderBeanConfig, spider, threadLocalData, throwable));
                }
            });
        }

        void requestAfter(List<SpiderBeanConfig> list, Connection connection) {
            list.forEach(spiderBeanConfig -> {
                List<SpiderMonitor> monitorList = spiderBeanConfig.getSpiderMonitorList();
                if (monitorList != null) {
                    monitorList.forEach(spiderMonitor -> spiderMonitor.requestAfter(connection, spiderBeanConfig, spider, threadLocalData));
                }
            });
        }

        void parseBefore(List<SpiderBeanConfig> list, Document document) {
            list.forEach(spiderBeanConfig -> {
                List<SpiderMonitor> monitorList = spiderBeanConfig.getSpiderMonitorList();
                if (monitorList != null) {
                    monitorList.forEach(spiderMonitor -> spiderMonitor.parseBefore(document, spiderBeanConfig, spider, threadLocalData));
                }
            });
        }

        void parseError(List<SpiderBeanConfig> list, Document document, Throwable throwable) {
            list.forEach(spiderBeanConfig -> {
                List<SpiderMonitor> monitorList = spiderBeanConfig.getSpiderMonitorList();
                if (monitorList != null) {
                    monitorList.forEach(spiderMonitor -> spiderMonitor.parseError(document, spiderBeanConfig, spider, threadLocalData, throwable));
                }
            });
        }

        void parseAfter(List<SpiderBeanConfig> list, Document document, Map<String, Object> parseData) {
            list.forEach(spiderBeanConfig -> {
                List<SpiderMonitor> monitorList = spiderBeanConfig.getSpiderMonitorList();
                if (monitorList != null) {
                    monitorList.forEach(spiderMonitor -> spiderMonitor.parseAfter(document, spiderBeanConfig, parseData, spider, threadLocalData));
                }
            });
        }

        Map<String, Object> parseHtml(SpiderBeanConfig spiderBeanConfig, Elements root) {
            SpiderBean tempBean = spiderBeanConfig.getSpiderBean();
            Map<String, Object> entity = new HashMap<>(tempBean.getFieldList().size());
            Elements roots = ObjectUtil.notBlank(tempBean.getSelector()) ? root.select(tempBean.getSelector()) : root;
            parseHtml(roots, new HashMap<>(16), tempBean, entity);

            return entity;
        }

        void parseHtml(Elements root, Map<String, Elements> elementContext, SpiderBean spiderBean, Map<String, Object> res) {
            spiderBean.getFieldList().forEach(spiderBeanField -> {
                if (spiderBeanField.getSpiderBean() != null) {
                    SpiderBean tempBean = spiderBeanField.getSpiderBean();
                    Map<String, Object> entity = new HashMap<>(tempBean.getFieldList().size());
                    Elements roots = ObjectUtil.notBlank(spiderBean.getSelector()) ? root.select(spiderBean.getSelector()) : root;
                    parseHtml(roots, elementContext, tempBean, entity);

                    Object obj = switchParseObjectToBean(entity, tempBean.getBeanClass(), spiderBeanField.isCollection());
                    res.put(spiderBeanField.getFieldName(), obj);
                } else {
                    List<Object> valueList;
                    if(spiderBeanField.isSetCurrentUrl()){
                        valueList = new ArrayList<>(3);
                        valueList.add(threadLocalData.getCurrentRequestUrl().url());
                    }else{
                        //复用文本
                        Elements elements = elementContext.get(spiderBeanField.getSelector());
                        if (elements == null) {
                            elements = root.select(spiderBeanField.getSelector());
                            elementContext.put(spiderBeanField.getSelector(), elements);
                        }
                        valueList = new ArrayList<>(elements.size());
                        elements.forEach(element -> {
                            String value;
                            if (spiderBeanField.getValueKey().equalsIgnoreCase("text")) {
                                value = element.text();
                            } else {
                                value = element.attr(spiderBeanField.getValueKey());
                            }
                            if (ObjectUtil.notBlank(value) && spiderBeanField.isUrl()) {
                                //url前置域名不全
                                value = PatternUtil.isUrl(value) ? value : threadLocalData.getCurrentRequestUrlDomain() + value;
                            }
                            valueList.add(value);
                        });
                    }
                    res.put(spiderBeanField.getFieldName(), spiderBeanField.isCollection() ? valueList : (valueList.isEmpty() ? "" : valueList.get(0)));

                    //是否为任务url并添加
                    isTaskUrlAndAdd(spiderBeanField ,valueList);
                }
            });
        }

        void isTaskUrlAndAdd(SpiderBeanField spiderBeanField ,List<Object> valueList){
            //是否为url爬取任务字段
            if (getUrlTotal() < spiderConfig.getMaxTaskCount() && spiderBeanField.isAddTaskUrl()) {
                List<HttpRequest> httpRequestList = new ArrayList<>(16);
                if (!valueList.isEmpty()) {
                    valueList.forEach(url -> {
                        String innerUrl = url.toString();
                        if (ObjectUtil.isBlank(PatternUtil.getUrlDomain(innerUrl))) {
                            innerUrl = threadLocalData.getCurrentRequestUrlDomain() + url;
                        }
                        if (threadLocalData.getCurrentRequestUrl().url().equals(innerUrl) || urlContains(innerUrl)) {
                            //同请求地址，跳过http://www.530p.com/
                            return;
                        }
                        if (PatternUtil.isUrl(innerUrl)) {
                            HttpRequest httpRequest = spiderBeanField.isPostRequest() ? new HttpPost(innerUrl) : new HttpGet(innerUrl);
                            httpRequestList.add(httpRequest);
                        }
                    });
                }

                if (!httpRequestList.isEmpty()) {
                    setUrl(httpRequestList);
                }
            }
        }

        Object switchParseObjectToBean(Map<String, Object> entity, Class<?> beanClass, boolean isCollection) {
            Object res = entity;
            if (isCollection) {
                List<Map<String, Object>> objectList = new ArrayList<>(16);
                res = objectList;
                entity.forEach((k, v) -> {
                    if (List.class.isAssignableFrom(v.getClass())) {
                        List list = (List) v;
                        for (int i = 0; i < list.size(); ++i) {
                            Map<String, Object> map;
                            if (objectList.size() <= i) {
                                map = new HashMap<>(16);
                                objectList.add(map);
                            } else {
                                map = objectList.get(i);
                            }
                            map.put(k, list.get(i));
                        }
                    } else {
                        if (objectList.isEmpty()) {
                            objectList.add(ObjectUtil.toMap(k, v));
                        } else {
                            objectList.forEach(stringObjectMap -> stringObjectMap.put(k, v));
                        }
                    }
                });
            }
            if (beanClass == null) {
                return res;
            }
            if(List.class.isAssignableFrom(res.getClass())){
                List<Object> resList = (List<Object>) res;
                List<Object> objectList = new ArrayList<>(resList.size());
                resList.forEach(o -> objectList.add(JSON.parseObject(JSON.toJSONString(o) ,beanClass)));
                return objectList;
            }
            return JSON.parseObject(JSON.toJSONString(res) ,beanClass);
        }
    }
}
